import random
import hashlib

urls = [f"https://www.baidu.com.{random.randint(1, 100)}" for i in range(100)]
print("去重前", len(urls))


# def md5(url):
#     return hashlib.md5(url.encode("utf-8")).hexdigest()
#
#
# urls = list(set(urls))
# print("去重后", len(urls))

differ_urls = []
fingerprint_set = set()
for url in urls:
    fingerprint = hashlib.md5(url.encode("utf-8")).hexdigest()
    if fingerprint not in fingerprint_set:
        differ_urls.append(url)
        fingerprint_set.add(fingerprint)
        print(fingerprint)
